package com.atbeijing.bigdata.spark.core.rdd.operator.transform

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Spark09_Oper_Transform {

    def main(args: Array[String]): Unit = {

        val conf = new SparkConf().setMaster("local[*]").setAppName("TransformOperator")
        val sc = new SparkContext(conf)

        // TODO 算子 - 转换 - distinct

        val rdd = sc.makeRDD(List(1,1))
        // shuffle
        // map(x => (x, null)).reduceByKey((x, _) => x).map(_._1)
        // 1, 1 => map(x => (x, null)) => (1, null), (1, null)
        // (1, null), (1, null) =>reduceByKey((x, _) => x) => (1, (null, null)) => (1, null)
        // (1, null) =>map(_._1) => 1
        // List(1,2,1,2).distinct (单点)
        val rdd1: RDD[Int] = rdd.distinct()
        //指定去重后的分区数
        val rdd2: RDD[Int] = rdd.distinct(5)
        rdd1.collect().foreach(println)//1

        sc.stop()

    }
}
